package com.lheia.process.zhejiang.quzhou


import com.lheia.jcb.spider.ApprovalPublicity
import com.lheia.jcb.spider.IpeSpiderData
import com.lheia.process.BaseProcess
import com.lheia.tool.ToolDateTime
import com.lheia.tool.ToolHttpConnect
import com.lheia.tool.ToolString
import com.lheia.tool.ToolUserAgents
import com.lheia.util.PageFieldConstants
import com.lheia.util.RequestExtraConstants
import com.lheia.util.ToolUrl
import org.jsoup.Jsoup
import us.codecraft.webmagic.Page
import us.codecraft.webmagic.Request
import us.codecraft.webmagic.Site
import us.codecraft.webmagic.model.HttpRequestBody
import us.codecraft.webmagic.processor.PageProcessor
import us.codecraft.webmagic.selector.HtmlNode
import us.codecraft.webmagic.utils.HttpConstant

/**
 * 广州黄埔区爬虫
 */
class SpiderSZFProcess implements BaseProcess {
    private Site site = Site.me().setCycleRetryTimes(3).setUserAgent(ToolUserAgents.userAgentRandom())
    private String province = "浙江省"
    private String city = "衢州市"
    private String county = ""
    private String approveDept = "市政府"

    private pageFormData = [
            infotypeId   : "",
            jdid         : "3084",
            area         : "113308000026182931",
            divid        : "div1529597",
            vc_title     : "",
            vc_number    : "",
            sortfield    : ",compaltedate:0",
            currpage     : "2",
            vc_filenumber: "",
            vc_all       : "",
            texttype     : "",
            fbtime       : "",
            texttype     : "",
            fbtime       : "",
            vc_all       : "",
            vc_filenumber: "",
            vc_title     : "",
            vc_number    : "",
            currpage     : 2,
            sortfield    : "", compaltedate: "0"]


    private pageQueryData = [[
                                     divid     : "div1529597",
                                     infotypeId: "A0705",
                                     jdid      : "3084",
                                     area      : "113308000026182931",
                                     sortfield : ""
                             ], [divid     : "div1529597",
                                 infotypeId: "A1710",
                                 jdid      : "3084",
                                 area      : "113308000026182931",
                                 sortfield : ""],
    ]

    void setIfTest(boolean ifTest) {
        this.ifTest = ifTest
    }
    protected boolean ifTest = true
    /**
     * process the page, extract urls to fetch, extract the data and store
     *
     * @param page page
     */
    @Override
    /**
     * 主要思路使用xpath解析列表也对象添加到请求队列中
     *
     */
    void process(Page page) {
        def methodName = page.getRequest().getExtra(RequestExtraConstants.CALLBACK)
        if (methodName) {
            this."$methodName"(page)
        } else {
            println(page)
        }
    }

    @Override
    def getBeginRequest(String url) {
        List<Request> list = new ArrayList<>()
        pageQueryData.each {
            url = ToolHttpConnect.appendUrlParam(url, it)
            Request request = new Request(url)
            /**
             * 使用无头浏览器
             */
//        request.putExtra(RequestExtraConstants.PHANTOM_JS,true)
            /**
             * 不计入去重
             */
            request.putExtra(RequestExtraConstants.NO_NEED_TO_REMOVE, true)
            request.setMethod(HttpConstant.Method.POST)
            /**
             * 处理回调
             */
            request.putExtra(RequestExtraConstants.CALLBACK, "processList")
            list << request
        }


        return list
    }

    /***
     * 列表处理
     * @param page
     */
    void processList(Page page) {
        /**列表node查找***/
        def trList = page.getHtml().getDocument().select("table.tb_main").select("tr:gt(0)")
        trList.each {
            def listNode = it.getElementsByTag("a").get(0)
            def detailUrl = listNode.attr("href")
            def title = listNode.attr("title")

            /***
             * 过滤title是否抓取抓取之后
             */
            def abs = ToolUrl.getAbsoluteURL(page.getUrl().toString(), detailUrl)
            def detailRequest = new Request(abs)
            println String.join(",", [title, abs])
            detailRequest.putExtra(RequestExtraConstants.CALLBACK, "processDetailsNPZ")
            page.addTargetRequest(detailRequest)
        }
        page.getResultItems().setSkip(true)
        processPaging(page)
    }


    /***
     * 拟批准
     */
    void processDetailsNPZ(Page page) {
        def resList = []
        def document = page.getHtml().getDocument()
        def approvalPublicity = new ApprovalPublicity()
        approvalPublicity.regionName = this.province
        approvalPublicity.regionCode = this.city
        approvalPublicity.county = this.county
        approvalPublicity.title =  document.select(
                "body > div.main > div > div > table:nth-child(4) > tbody > tr > td > table:nth-child(1) > tbody > tr > td")
                .text()
        approvalPublicity.fileContent =  ToolString.getTextFromHtml(document.select("body > div.main > div > div > table:nth-child(4) > tbody > tr > td > table:nth-child(3) > tbody > tr > td").text())
        approvalPublicity.projectName= ToolString.matchList(approvalPublicity.fileContent,"(?<=项目名称：)(.*)(?=项目建设地点|建设地点)")
        approvalPublicity.projectLoc =ToolString.matchList(approvalPublicity.fileContent,"(?<=项目建设地点：|建设地点：)(.*?)(?=建设单位)")
        approvalPublicity.developmentOrg =ToolString.matchList(approvalPublicity.fileContent,"(?<=建设单位：)(.*?)(?=建设项目概况|环评机构)")
        approvalPublicity.fileId =  document.select("body > div.main > div > div > table:nth-child(4) > tbody > tr > td > table:nth-child(3) > tbody > tr > td").select("a[href]").attr("href")
        approvalPublicity.pubDate = ToolDateTime.extractDate(document.select("body > div.main > div > div > div > table > tbody > tr:nth-child(1) > td:nth-child(4)").text())
        page.putField(PageFieldConstants.RESULT_DATA, approvalPublicity)
    }
    /**
     * 分页处理
     * @param page
     */
    void processPaging(Page page) {
        /***
         * 获取总页数
         */
        def pageTotalNum = ToolString.matchList(page
                .getHtml().getDocument().select("table.tb_title")
                .select(":containsOwn(记录)").text(), '\\d+').get(1)
        pageTotalNum = pageTotalNum ? Integer.valueOf(pageTotalNum) : 0
        def pevReq = page.getRequest()
        def formData = pevReq.getExtra(RequestExtraConstants.PREV_FORM_DATA)
        if (formData) {
            formData.currpage = formData.currpage + 1
        } else {
            formData = this.pageFormData
        }
        /**
         * 若小于等于总页数则翻页
         */
        if (formData.currpage <= pageTotalNum) {
            def url = pevReq.getUrl()
            def pageRequest = new Request(url)
            pageRequest.setMethod(HttpConstant.Method.POST)
            pageRequest.putExtra(RequestExtraConstants.PREV_FORM_DATA, formData)
            /***
             * 分页请求不需要进行去重
             */
            pageRequest.putExtra(RequestExtraConstants.NO_NEED_TO_REMOVE, true)
            /**
             * 是否使用phantomJS
             */
            pageRequest.putExtra(RequestExtraConstants.PHANTOM_JS, page.getRequest().getExtra(RequestExtraConstants.PHANTOM_JS))
            pageRequest.setRequestBody(HttpRequestBody.form(formData, ToolString.UTF_8))
            /**
             * 处理回调
             */
            pageRequest.putExtra(RequestExtraConstants.CALLBACK, "processList")
            println("下一页," + pageRequest)
            page.addTargetRequest(pageRequest)
        }
    }

    /**
     * 建设项目
     */
    def processDetailsHP(Page page) {
        println(page)
        def resList = []
        def doc = page.getHtml().getDocument()
        def dataMap = [:]
        dataMap.regionName = this.province
        dataMap.regionCode = this.city
        dataMap.county = this.county
        dataMap.approveDept = this.approveDept

        /**获取标题**/
        def htmlHead = doc.select("div.content_xxgkAttr").select("li")
        dataMap.pubDate = ToolDateTime.extractDate(htmlHead?.get(3).toString())
        dataMap.title = doc.select("h1.content_title")
        dataMap.spiderFileUrl = page.getUrl()

        /**获取列表内容**/
        def trList = doc.select("table").select("tr:gt(1)")
        for (def trNode : trList) {
            def tdList = trNode.select("td")
            dataMap.projectName = tdList.get(0).text()
            dataMap.approveDate = ToolDateTime.extractDate(tdList.get(1).text())
            def approvalPublicity = new ApprovalPublicity()
            approvalPublicity.properties = dataMap
            resList << approvalPublicity
        }
        page.putField(PageFieldConstants.RESULT_DATA, resList)
    }
    /**
     * 如果为测试环境或者开发环境不计入redis去重只使用redisList队列
     * @param page
     */
    void processTargetResquest(Page page) {
        if (ifTest) {
            println(page)
            page.getTargetRequests().each {
                it.putExtra(RequestExtraConstants.NO_NEED_TO_REMOVE, true)
            }
        }
    }


/**
 * get the site settings
 *
 * @return site* @see Site
 */
    @Override
    Site getSite() {
        return site
    }
}
